const request = require('request-promise');
const cheerio = require('cheerio');
const axios = require('axios');
const fs = require('fs');

class Crawler {
    constructor(options = {}) {
        this.options = options;
    }

    async fetch(url, formatFn = null) {
        const requestOptions = {
            url: url,
            headers: this.options.headers || {},
            timeout: this.options.timeout || 5000,
            proxy: this.options.proxy || '',
            // resolveWithFullResponse: true,
        };

        let response;
        try {
            // response = await request(requestOptions);
            response = await axios(requestOptions);
        } catch (err) {
            throw new Error(`网页获取失败: ${url}. Error: ${err.message}`);
        }
        const $ = cheerio.load(response.data);
        const data = formatFn ? formatFn($, response) : $;

        return {url, data};
    }

    async fetchAll(urls, formatFn = null) {
        const results = [];
        for (let url of urls) {
            try {
                const result = await this.fetch(url, formatFn);
                results.push(result);
            } catch (err) {
                throw new Error(`网页获取失败: ${url}. Error: ${err.message}`);
            }
        }
        return {url: urls, data: results};
    }
}

// Demo

/*const crawler = new Crawler({
    headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
    },
    timeout: 5000,
    proxy: '',
});

(async () => {
    try {
        const result = await crawler.fetch('https://gy.lianjia.com/ershoufang/pg3co32/', ($, response) => {
            // const title = $('.house-item').text();
            const statusCode = response.status;
            // 使用 Cheerio 提取房源数据
            const houseList = [];
            $('.sellListContent li').each((index, element) => {
                const img = $(element).find('.lj-lazy').attr('data-original');
                const title = $(element).find('.title a').text().trim();
                const position = $(element).find('.positionInfo').text().trim();
                const houseInfo = $(element).find('.houseInfo').text().trim();
                const price = $(element).find('.totalPrice span').text().trim();
                // const unitPrice = $(element).find('.unitPrice').find('span').text();
                const unitPrice = $(element).find('.unitPrice').attr('data-price')

                if (title) {
                    // 创建房源对象并添加到列表中
                    const house = {
                        title,
                        position,
                        houseInfo,
                        price,
                        unitPrice,
                        img,
                    };
                    houseList.push(house);
                }

            })
            // houseList.forEach((house, index) => {
            //     console.log(`房源 ${index + 1}:`);
            //     console.log(`标题: ${house.title}`);
            //     console.log(`位置: ${house.position}`);
            //     console.log(`价格: ${house.price}`);
            //     console.log(`单价: ${house.unitPrice}`);
            //     console.log(`图片: ${house.img}`);
            //     console.log(`结构: ${house.houseInfo}`);
            //     console.log('------------------------------');
            // });
            console.log(houseList)
            return {houseList, statusCode};
        });
        // console.table(result);
    } catch (err) {
        console.error(err);
    }
})();*/

// 生成贵阳随机经纬度
function getRandomCoordinates() {
    const minLat = 26.4254; // 最小纬度
    const maxLat = 26.7551; // 最大纬度
    const minLng = 106.5207; // 最小经度
    const maxLng = 106.7974; // 最大经度

    const lat = Math.random() * (maxLat - minLat) + minLat;
    const lng = Math.random() * (maxLng - minLng) + minLng;

    return {lat, lng};
}

// 根据经纬度解析地址
async function getLocationInfo() {
    let info = await axios({
        url: `https://apis.map.qq.com/ws/geocoder/v1/?key=QULBZ-ERHWG-2YXQ6-IP3UZ-UCVVE-VFBW4&&location=${getRandomCoordinates().lat},${getRandomCoordinates().lng}`,
        method: 'get'
    })

    if (info.data) {
        let D = info.data.result
        let p = {
            "provinceId": "520000",
            "province": "贵州省",
            "streetId": D.ad_info.city_code,
            "street": D.address_component.district,
            "longitude": D.location.lng,
            "latitude": D.location.lat,
            "positionDetail": D.address,
            "cityId": D.ad_info.city_code,
            "city": D.address_component.city,
        }
        console.log(p)
        return p
    }

}


const crawler = new Crawler({
    headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
    },
    timeout: 5000,
    proxy: '',
});

(async () => {
    try {
        let lo =  await getLocationInfo()
         const result = await crawler.fetchAll(
             [
                 'https://dl.lianjia.com/ershoufang/bayilu/',
                 'https://dl.lianjia.com/ershoufang/bayilu/pg2/',
                 'https://dl.lianjia.com/ershoufang/bayilu/pg3/'], ($, response) => {
                 // const title = $('.house-item').text();
                 const statusCode = response.status;
                 // 使用 Cheerio 提取房源数据
                 const houseList = [];
                 $('.sellListContent li').each((index, element) => {
                     const imgs = $(element).find('.lj-lazy').attr('data-original');
                     const title = $(element).find('.title a').text().trim();
                     const position = $(element).find('.positionInfo').text().trim();
                     const houseInfo = $(element).find('.houseInfo').text().trim();
                     const price = $(element).find('.totalPrice span').text().trim();
                     // const unitPrice = $(element).find('.unitPrice').find('span').text();
                     const unitPrice = $(element).find('.unitPrice').attr('data-price')
                     const houseInfoParts = houseInfo.split('|').map(part => part.trim());
                     const [structure, floorArea, direction, decoration, floor, buildStructure] = houseInfoParts;

                     if (title) {
                         // 创建房源对象并添加到列表中
                         const house = {
                             "w": "1",
                             "userId": "77e41ae1-a0f1-410c-93f5-a792945e6c10",
                             "unitPrice": unitPrice,
                             "type": 0,
                             "trafficInfo": title,
                             "title": title,
                             "tags": [
                                 "环境优美",
                                 "随时看房",
                                 "配套齐全"
                             ],
                             "t": "2",
                             "suitablePopulation": title,
                             "structure": structure,

                             "s": "3",

                             "price": price,
                             "positionDetail": position,
                             "phone": "15186817855",
                             "partNum": "3",
                             "name": "周义",
                             "longitude": 106.67897,
                             "latitude": 26.559692,
                             "innerArea": "110",
                             "imgs": [imgs || 'https://image1.ljcdn.com/110000-inspection/pc1_RhVVWu7GK_1.jpg.280x210.jpg'],
                             "houseTypeStructure": "平层",
                             "houseTypeIntroduction": title,
                             "houseSellingIntroduction": title,
                             "houseNum": "3305",
                             "floorArea": floorArea.replace('平米', ''),
                             "floor": floor,
                             "email": "1840354092@qq.com",
                             "elevatorOfHouse": "1梯4户",
                             "elevator": true,
                             "direction": direction,
                             "decoration": decoration,
                             "coreSellingInfo": title,
                             "cityId": "210200",
                             "city": "大连市",
                             "streetId": "210203010",
                             "street": "八一路街道",
                             "areaId": "210203",
                             "area": "西岗区",
                             "provinceId": "210000",
                             "province": "辽宁省",
                             "c": "1",
                             "buildStructure": buildStructure,
                             "buildingNum": Math.ceil(Math.random() * 10),
                             "aroundInfo": title,
                             "application": "普通住宅"
                         };
                         houseList.push(house);
                     }

                 })

                 // 将数据转换为 JSON 格式
                 const jsonData = JSON.stringify(houseList, null, 2);
 // 写入文件
                 fs.appendFile('data.json', jsonData, 'utf8', (err) => {
                     if (err) {
                         console.error('写入文件出错:', err);
                         return;
                     }
                     console.log('数据已写入到 data.json 文件');
                 });
                 return {houseList, statusCode};
             });
        // console.table(result);
    } catch (err) {
        console.error(err);
    }
})();


module.exports = Crawler;


